<!-- build time:Wed Jun 21 2023 22:33:34 GMT+0800 (GMT+08:00) --><!DOCTYPE html><html lang="zh-CN"><head><meta charset="UTF-8"><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=2"><meta name="theme-color" content="#FFF"><meta name="baidu-site-verification" content="code-C0oocRvMWv"><link rel="apple-touch-icon" sizes="180x180" href="/images/apple-touch-icon.png"><link rel="icon" type="image/ico" sizes="32x32" href="/images/favicon.ico"><link rel="mask-icon" href="/images/logo.svg" color=""><link rel="manifest" href="/images/manifest.json"><meta name="msapplication-config" content="/images/browserconfig.xml"><meta http-equiv="Cache-Control" content="no-transform"><meta http-equiv="Cache-Control" content="no-siteapp"><meta name="baidu-site-verification" content="https://jiang-hs.gitee.io"><link rel="alternate" type="application/rss+xml" title="航 順" href="https://jiang-hs.gitee.io/rss.xml"><link rel="alternate" type="application/atom+xml" title="航 順" href="https://jiang-hs.gitee.io/atom.xml"><link rel="alternate" type="application/json" title="航 順" href="https://jiang-hs.gitee.io/feed.json"><link rel="stylesheet" href="//fonts.googleapis.com/css?family=Mulish:300,300italic,400,400italic,700,700italic%7CFredericka%20the%20Great:300,300italic,400,400italic,700,700italic%7CNoto%20Serif%20JP:300,300italic,400,400italic,700,700italic%7CNoto%20Serif%20SC:300,300italic,400,400italic,700,700italic%7CInconsolata:300,300italic,400,400italic,700,700italic&display=swap&subset=latin,latin-ext"><link rel="stylesheet" href="/css/app.css?v=0.0.0"><link rel="canonical" href="https://jiang-hs.gitee.io/posts/898b5689/"><meta name="description" content="强化学习是一个通过奖惩来学习正确行为的机制。 家族中有很多种不一样的成员，有学习奖惩值，根据自己认为的高价值选行为， 比如 Q learning, Deep Q Network, 也有不通过分析奖励值，直接输出行为的方法，这就是今天要说的 Policy Gradient 了。甚至我们可以为 Policy Gradients 加上一个神经网络来输出预测的动作。对比起以值为基础的方法，Policy G"><meta property="og:type" content="article"><meta property="og:title" content="强化学习之 Policy Gradient"><meta property="og:url" content="https://jiang-hs.gitee.io/posts/898b5689/index.html"><meta property="og:site_name" content="航 順"><meta property="og:description" content="强化学习是一个通过奖惩来学习正确行为的机制。 家族中有很多种不一样的成员，有学习奖惩值，根据自己认为的高价值选行为， 比如 Q learning, Deep Q Network, 也有不通过分析奖励值，直接输出行为的方法，这就是今天要说的 Policy Gradient 了。甚至我们可以为 Policy Gradients 加上一个神经网络来输出预测的动作。对比起以值为基础的方法，Policy G"><meta property="og:locale" content="zh_CN"><meta property="og:image" content="https://shun309.oss-cn-hangzhou.aliyuncs.com/photos/20210624230118.png"><meta property="og:image" content="https://shun309.oss-cn-hangzhou.aliyuncs.com/photos/20210624230826.png"><meta property="og:image" content="https://shun309.oss-cn-hangzhou.aliyuncs.com/photos/20210624231131.png"><meta property="og:image" content="https://shun309.oss-cn-hangzhou.aliyuncs.com/photos/1494659-20191204201909905-1660812647.png"><meta property="og:image" content="https://shun309.oss-cn-hangzhou.aliyuncs.com/photos/1494659-20191204201814509-856709770.png"><meta property="og:image" content="https://shun309.oss-cn-hangzhou.aliyuncs.com/photos/output.gif"><meta property="og:image" content="https://shun309.oss-cn-hangzhou.aliyuncs.com/photos/20210625113142.png"><meta property="article:published_time" content="2021-06-24T14:58:45.000Z"><meta property="article:modified_time" content="2023-01-12T04:19:43.143Z"><meta property="article:author" content="hang shun"><meta property="article:tag" content="hang shun"><meta name="twitter:card" content="summary"><meta name="twitter:image" content="https://shun309.oss-cn-hangzhou.aliyuncs.com/photos/20210624230118.png"><title>强化学习之 Policy Gradient | hang shun = 航 順 = 天官赐福，百无禁忌</title><meta name="generator" content="Hexo 5.4.2"></head><body itemscope itemtype="http://schema.org/WebPage"><div id="loading"><div class="cat"><div class="body"></div><div class="head"><div class="face"></div></div><div class="foot"><div class="tummy-end"></div><div class="bottom"></div><div class="legs left"></div><div class="legs right"></div></div><div class="paw"><div class="hands left"></div><div class="hands right"></div></div></div></div><div id="container"><header id="header" itemscope itemtype="http://schema.org/WPHeader"><div class="inner"><div id="brand"><div class="pjax"><h1 itemprop="name headline">强化学习之 Policy Gradient</h1><div class="meta"><span class="item" title="创建时间：2021-06-24 22:58:45"><span class="icon"><i class="ic i-calendar"></i> </span><span class="text">发表于</span> <time itemprop="dateCreated datePublished" datetime="2021-06-24T22:58:45+08:00">2021-06-24</time> </span><span class="item" title="本文字数"><span class="icon"><i class="ic i-pen"></i> </span><span class="text">本文字数</span> <span>6.3k</span> <span class="text">字</span> </span><span class="item" title="阅读时长"><span class="icon"><i class="ic i-clock"></i> </span><span class="text">阅读时长</span> <span>6 分钟</span></span></div></div></div><nav id="nav"><div class="inner"><div class="toggle"><div class="lines" aria-label="切换导航栏"><span class="line"></span> <span class="line"></span> <span class="line"></span></div></div><ul class="menu"><li class="item title"><a href="/" rel="start">hang shun</a></li></ul><ul class="right"><li class="item theme"><i class="ic i-sun"></i></li><li class="item search"><i class="ic i-search"></i></li></ul></div></nav></div><div id="imgs" class="pjax"><ul><li class="item" data-background-image="https://pic1.imgdb.cn/item/64427ca20d2dde5777b04d90.png"></li><li class="item" data-background-image="https://pic1.imgdb.cn/item/60d7f99b5132923bf8aa654e.jpg"></li><li class="item" data-background-image="https://pic1.imgdb.cn/item/64427b920d2dde5777ae9810.jpg"></li><li class="item" data-background-image="https://pic1.imgdb.cn/item/64427a840d2dde5777acafa6.jpg"></li><li class="item" data-background-image="https://pic1.imgdb.cn/item/60d7f95f5132923bf8a924cb.jpg"></li><li class="item" data-background-image="https://pic1.imgdb.cn/item/64427c390d2dde5777afabd1.jpg"></li></ul></div></header><div id="waves"><svg class="waves" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 24 150 28" preserveAspectRatio="none" shape-rendering="auto"><defs><path id="gentle-wave" d="M-160 44c30 0 58-18 88-18s 58 18 88 18 58-18 88-18 58 18 88 18 v44h-352z"/></defs><g class="parallax"><use xlink:href="#gentle-wave" x="48" y="0"/><use xlink:href="#gentle-wave" x="48" y="3"/><use xlink:href="#gentle-wave" x="48" y="5"/><use xlink:href="#gentle-wave" x="48" y="7"/></g></svg></div><main><div class="inner"><div id="main" class="pjax"><div class="article wrap"><div class="breadcrumb" itemscope itemtype="https://schema.org/BreadcrumbList"><i class="ic i-home"></i> <span><a href="/">首页</a></span></div><article itemscope itemtype="http://schema.org/Article" class="post block" lang="zh-CN"><link itemprop="mainEntityOfPage" href="https://jiang-hs.gitee.io/posts/898b5689/"><span hidden itemprop="author" itemscope itemtype="http://schema.org/Person"><meta itemprop="image" content="/images/avatar.jpg"><meta itemprop="name" content="hang shun"><meta itemprop="description" content="天官赐福，百无禁忌, 世中逢尔，雨中逢花"></span><span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization"><meta itemprop="name" content="航 順"></span><div class="body md" itemprop="articleBody"><p>强化学习是一个通过奖惩来学习正确行为的机制。 家族中有很多种不一样的成员，有学习奖惩值，根据自己认为的高价值选行为， 比如 Q learning, Deep Q Network, 也有不通过分析奖励值，直接输出行为的方法，这就是今天要说的 Policy Gradient 了。甚至我们可以为 Policy Gradients 加上一个神经网络来输出预测的动作。对比起以值为基础的方法，Policy Gradients 直接输出动作的最大好处就是，它能在一个连续区间内挑选动作，而基于值的，比如 Q-learning，它如果在无穷多的动作中计算价值，从而选择行为，这，它可吃不消。</p><h1 id="1-什么是policy"><a class="anchor" href="#1-什么是policy">#</a> 1 什么是 Policy</h1><p>首先大致说明一下强化学习的基本结构：</p><p><img data-src="https://shun309.oss-cn-hangzhou.aliyuncs.com/photos/20210624230118.png" alt=""></p><p>如上图，一个由<em> Agent</em>（相当于我们的模型）和<em> Environment</em>（所处状态）组成的结构。</p><p><em>Agent</em> 通过观察当前环境的状态<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><msub><mi>s</mi><mi>t</mi></msub></mrow><annotation encoding="application/x-tex">s_t</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:.58056em;vertical-align:-.15em"></span><span class="mord"><span class="mord mathnormal">s</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:.2805559999999999em"><span style="top:-2.5500000000000003em;margin-left:0;margin-right:.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">t</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:.15em"><span></span></span></span></span></span></span></span></span></span>，得出当前应当执行的动作<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><msub><mi>a</mi><mi>t</mi></msub></mrow><annotation encoding="application/x-tex">a_t</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:.58056em;vertical-align:-.15em"></span><span class="mord"><span class="mord mathnormal">a</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:.2805559999999999em"><span style="top:-2.5500000000000003em;margin-left:0;margin-right:.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">t</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:.15em"><span></span></span></span></span></span></span></span></span></span>。Agent 执行完动作之后环境对应发生了改变，并且环境会给予<em> Agent</em> 一个反馈<em> reward</em> <span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><msub><mi>r</mi><mi>t</mi></msub></mrow><annotation encoding="application/x-tex">r_t</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:.58056em;vertical-align:-.15em"></span><span class="mord"><span class="mord mathnormal" style="margin-right:.02778em">r</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:.2805559999999999em"><span style="top:-2.5500000000000003em;margin-left:-.02778em;margin-right:.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">t</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:.15em"><span></span></span></span></span></span></span></span></span></span>。此时又会是一个新的环境状态 <span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><msup><mi>s</mi><mo mathvariant="normal" lspace="0em" rspace="0em">′</mo></msup></mrow><annotation encoding="application/x-tex">s&#x27;</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:.751892em;vertical-align:0"></span><span class="mord"><span class="mord mathnormal">s</span><span class="msupsub"><span class="vlist-t"><span class="vlist-r"><span class="vlist" style="height:.751892em"><span style="top:-3.063em;margin-right:.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mtight">′</span></span></span></span></span></span></span></span></span></span></span></span>，基于本次的环境状态，<em>Agent</em> 又会执行对应的动作... 以此类推持续进行下去，直到无法继续。</p><p>如下图所示，Env 表示环境，Actor 即为 Agent：<img data-src="https://shun309.oss-cn-hangzhou.aliyuncs.com/photos/20210624230826.png" alt=""></p><blockquote><p>上面实际上就是对一系列操作进行了抽象描述。如果以玩游戏为例说明，我们（<em>Agent</em>）通过观察游戏（<em>Environment</em>）的运行情况（<em>State</em>），选择接下来要执行的操作（<em>Action</em>），游戏往往还会反馈给我们我们的得分（<em>Rewards</em>）。</p></blockquote><p><strong>在不同的状态（<em>State</em>）采取的动作 <em>Action</em> 也就是我们所说的策略 <em>Policy</em> 。</strong> 常用符号<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>π</mi></mrow><annotation encoding="application/x-tex">π</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:.43056em;vertical-align:0"></span><span class="mord mathnormal" style="margin-right:.03588em">π</span></span></span></span> 来表示策略。</p><h1 id="2-算法思想"><a class="anchor" href="#2-算法思想">#</a> 2 算法思想</h1><p><strong>Policy Gradient 不通过误差反向传播，它通过观测信息选出一个行为直接进行反向传播</strong>，当然出人意料的是他并没有误差，而是利用 reward 奖励直接对选择行为的可能性进行增强和减弱，好的行为会被增加下一次被选中的概率，不好的行为会被减弱下次被选中的概率。<br>举例如下图所示：输入当前的状态，输出 action 的概率分布，选择概率最大的一个 action 作为要执行的操作。<br><img data-src="https://shun309.oss-cn-hangzhou.aliyuncs.com/photos/20210624231131.png" alt=""></p><p>而一个完整的策略 <span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>τ</mi></mrow><annotation encoding="application/x-tex">τ</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:.43056em;vertical-align:0"></span><span class="mord mathnormal" style="margin-right:.1132em">τ</span></span></span></span> 代表的是一整个回合中，对于每个状态下所采取的的动作所构成的序列，而每个回合 episode 中每个动作的回报和等于一个回合的回报值 <span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>R</mi><mo>=</mo><msubsup><mo>∑</mo><mrow><mi>t</mi><mo>=</mo><mn>1</mn></mrow><mi>T</mi></msubsup><msub><mi>r</mi><mi>t</mi></msub></mrow><annotation encoding="application/x-tex">R = ∑ _{t = 1} ^T r_t</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:.68333em;vertical-align:0"></span><span class="mord mathnormal" style="margin-right:.00773em">R</span><span class="mspace" style="margin-right:.2777777777777778em"></span><span class="mrel">=</span><span class="mspace" style="margin-right:.2777777777777778em"></span></span><span class="base"><span class="strut" style="height:1.2809409999999999em;vertical-align:-.29971000000000003em"></span><span class="mop"><span class="mop op-symbol small-op" style="position:relative;top:-.0000050000000000050004em">∑</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:.981231em"><span style="top:-2.40029em;margin-left:0;margin-right:.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mathnormal mtight">t</span><span class="mrel mtight">=</span><span class="mord mtight">1</span></span></span></span><span style="top:-3.2029em;margin-right:.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight" style="margin-right:.13889em">T</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:.29971000000000003em"><span></span></span></span></span></span></span><span class="mspace" style="margin-right:.16666666666666666em"></span><span class="mord"><span class="mord mathnormal" style="margin-right:.02778em">r</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:.2805559999999999em"><span style="top:-2.5500000000000003em;margin-left:-.02778em;margin-right:.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">t</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:.15em"><span></span></span></span></span></span></span></span></span></span></p><p><img data-src="https://shun309.oss-cn-hangzhou.aliyuncs.com/photos/1494659-20191204201909905-1660812647.png" alt=""></p><p><strong>Trajectory <span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>τ</mi></mrow><annotation encoding="application/x-tex">τ</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:.43056em;vertical-align:0"></span><span class="mord mathnormal" style="margin-right:.1132em">τ</span></span></span></span></strong> ：行动 action 和状态 state 的序列</p><p><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><msub><mi>p</mi><mi>θ</mi></msub><mo stretchy="false">(</mo><mi>τ</mi><mo stretchy="false">)</mo></mrow><annotation encoding="application/x-tex">p_θ(τ)</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-.25em"></span><span class="mord"><span class="mord mathnormal">p</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:.33610799999999996em"><span style="top:-2.5500000000000003em;margin-left:0;margin-right:.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight" style="margin-right:.02778em">θ</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:.15em"><span></span></span></span></span></span></span><span class="mopen">(</span><span class="mord mathnormal" style="margin-right:.1132em">τ</span><span class="mclose">)</span></span></span></span> ：<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>π</mi></mrow><annotation encoding="application/x-tex">π</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:.43056em;vertical-align:0"></span><span class="mord mathnormal" style="margin-right:.03588em">π</span></span></span></span> 在参数为<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>θ</mi></mrow><annotation encoding="application/x-tex">θ</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:.69444em;vertical-align:0"></span><span class="mord mathnormal" style="margin-right:.02778em">θ</span></span></span></span> 情况下时 <span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>τ</mi></mrow><annotation encoding="application/x-tex">τ</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:.43056em;vertical-align:0"></span><span class="mord mathnormal" style="margin-right:.1132em">τ</span></span></span></span> 发生的概率</p><p>得到了概率之后我们就可以根据采样得到的回报值计算出数学期望，从而得到目标函数，然后用来更新我们的参数 θ</p><p><img data-src="https://shun309.oss-cn-hangzhou.aliyuncs.com/photos/1494659-20191204201814509-856709770.png" alt=""></p><p>得出目标函数之后，就需要根据目标函数求解目标函数最大值以及最大值对应的 policy 的参数 θ。类比深度学习中的梯度下降求最小值的方法，由于我们这里需要求的是<strong>目标函数的最大值</strong>，因此需要采取的方法是<strong>梯度上升</strong>。也就是说，思想起点是一样的，即需要求出目标函数的梯度。</p><p>优点：</p><ul><li>连续的动作空间（或者高维空间）中更加高效；</li><li>可以实现随机化的策略；</li><li>某种情况下，价值函数可能比较难以计算，而策略函数较容易。</li></ul><p>缺点：</p><ul><li><p>通常收敛到局部最优而非全局最优</p></li><li><p>评估一个策略通常低效（这个过程可能慢，但是具有更高的可变性，其中也会出现很多并不有效的尝试，而且方差高</p></li></ul><h1 id="3-策略函数"><a class="anchor" href="#3-策略函数">#</a> 3 策略函数</h1><p>在 Policy Based 强化学习方法下，我们对策略进行近似表示。此时策略<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>π</mi></mrow><annotation encoding="application/x-tex">π</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:.43056em;vertical-align:0"></span><span class="mord mathnormal" style="margin-right:.03588em">π</span></span></span></span> 可以被描述为一个包含参数<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>θ</mi></mrow><annotation encoding="application/x-tex">θ</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:.69444em;vertical-align:0"></span><span class="mord mathnormal" style="margin-right:.02778em">θ</span></span></span></span> 的函数，即：</p><p><span class="katex-display"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML" display="block"><semantics><mrow><msub><mi>π</mi><mi>θ</mi></msub><mo stretchy="false">(</mo><mi>s</mi><mo separator="true">,</mo><mi>a</mi><mo stretchy="false">)</mo><mo>=</mo><mi>P</mi><mo stretchy="false">(</mo><mi>a</mi><mi mathvariant="normal">∣</mi><mi>s</mi><mo separator="true">,</mo><mi>θ</mi><mo stretchy="false">)</mo><mo>≈</mo><mi>π</mi><mo stretchy="false">(</mo><mi>a</mi><mi mathvariant="normal">∣</mi><mi>s</mi><mo stretchy="false">)</mo></mrow><annotation encoding="application/x-tex">π_θ(s,a)=P(a|s,θ)≈π(a|s)</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-.25em"></span><span class="mord"><span class="mord mathnormal" style="margin-right:.03588em">π</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:.33610799999999996em"><span style="top:-2.5500000000000003em;margin-left:-.03588em;margin-right:.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight" style="margin-right:.02778em">θ</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:.15em"><span></span></span></span></span></span></span><span class="mopen">(</span><span class="mord mathnormal">s</span><span class="mpunct">,</span><span class="mspace" style="margin-right:.16666666666666666em"></span><span class="mord mathnormal">a</span><span class="mclose">)</span><span class="mspace" style="margin-right:.2777777777777778em"></span><span class="mrel">=</span><span class="mspace" style="margin-right:.2777777777777778em"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-.25em"></span><span class="mord mathnormal" style="margin-right:.13889em">P</span><span class="mopen">(</span><span class="mord mathnormal">a</span><span class="mord">∣</span><span class="mord mathnormal">s</span><span class="mpunct">,</span><span class="mspace" style="margin-right:.16666666666666666em"></span><span class="mord mathnormal" style="margin-right:.02778em">θ</span><span class="mclose">)</span><span class="mspace" style="margin-right:.2777777777777778em"></span><span class="mrel">≈</span><span class="mspace" style="margin-right:.2777777777777778em"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-.25em"></span><span class="mord mathnormal" style="margin-right:.03588em">π</span><span class="mopen">(</span><span class="mord mathnormal">a</span><span class="mord">∣</span><span class="mord mathnormal">s</span><span class="mclose">)</span></span></span></span></span></p><p>我们现在来看<strong>策略函数</strong><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><msub><mi>π</mi><mi>θ</mi></msub><mo stretchy="false">(</mo><mi>s</mi><mo separator="true">,</mo><mi>a</mi><mo stretchy="false">)</mo></mrow><annotation encoding="application/x-tex">π_θ(s,a)</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-.25em"></span><span class="mord"><span class="mord mathnormal" style="margin-right:.03588em">π</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:.33610799999999996em"><span style="top:-2.5500000000000003em;margin-left:-.03588em;margin-right:.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight" style="margin-right:.02778em">θ</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:.15em"><span></span></span></span></span></span></span><span class="mopen">(</span><span class="mord mathnormal">s</span><span class="mpunct">,</span><span class="mspace" style="margin-right:.16666666666666666em"></span><span class="mord mathnormal">a</span><span class="mclose">)</span></span></span></span> 的设计：</p><p>最常用的策略函数就是<strong> softmax 策略函数</strong>了，它主要应用于离散空间中，softmax 策略使用描述状态和行为的特征<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>ϕ</mi><mo stretchy="false">(</mo><mi>s</mi><mo separator="true">,</mo><mi>a</mi><mo stretchy="false">)</mo></mrow><annotation encoding="application/x-tex">ϕ(s,a)</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-.25em"></span><span class="mord mathnormal">ϕ</span><span class="mopen">(</span><span class="mord mathnormal">s</span><span class="mpunct">,</span><span class="mspace" style="margin-right:.16666666666666666em"></span><span class="mord mathnormal">a</span><span class="mclose">)</span></span></span></span> 与参数<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>θ</mi></mrow><annotation encoding="application/x-tex">θ</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:.69444em;vertical-align:0"></span><span class="mord mathnormal" style="margin-right:.02778em">θ</span></span></span></span> 的线性组合来权衡一个行为发生的几率，即:</p><p><span class="katex-display"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML" display="block"><semantics><mrow><msub><mi>π</mi><mi>θ</mi></msub><mo stretchy="false">(</mo><mi>s</mi><mo separator="true">,</mo><mi>a</mi><mo stretchy="false">)</mo><mo>=</mo><mfrac><msup><mi>e</mi><mrow><mi>ϕ</mi><mo stretchy="false">(</mo><mi>s</mi><mo separator="true">,</mo><mi>a</mi><msup><mo stretchy="false">)</mo><mi>T</mi></msup><mi>θ</mi></mrow></msup><mrow><munder><mo>∑</mo><mi>b</mi></munder><msup><mi>e</mi><mrow><mi>ϕ</mi><mo stretchy="false">(</mo><mi>s</mi><mo separator="true">,</mo><mi>b</mi><msup><mo stretchy="false">)</mo><mi>T</mi></msup><mi>θ</mi></mrow></msup></mrow></mfrac></mrow><annotation encoding="application/x-tex">π_θ(s,a)=\frac{e^{ϕ(s,a)^Tθ}}{ \sum_{b}e^{ϕ(s,b)^Tθ}}</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-.25em"></span><span class="mord"><span class="mord mathnormal" style="margin-right:.03588em">π</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:.33610799999999996em"><span style="top:-2.5500000000000003em;margin-left:-.03588em;margin-right:.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight" style="margin-right:.02778em">θ</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:.15em"><span></span></span></span></span></span></span><span class="mopen">(</span><span class="mord mathnormal">s</span><span class="mpunct">,</span><span class="mspace" style="margin-right:.16666666666666666em"></span><span class="mord mathnormal">a</span><span class="mclose">)</span><span class="mspace" style="margin-right:.2777777777777778em"></span><span class="mrel">=</span><span class="mspace" style="margin-right:.2777777777777778em"></span></span><span class="base"><span class="strut" style="height:2.7039400000000002em;vertical-align:-1.020575em"></span><span class="mord"><span class="mopen nulldelimiter"></span><span class="mfrac"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:1.683365em"><span style="top:-2.2855em"><span class="pstrut" style="height:3.0063649999999997em"></span><span class="mord"><span class="mop"><span class="mop op-symbol small-op" style="position:relative;top:-.0000050000000000050004em">∑</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:.1863979999999999em"><span style="top:-2.40029em;margin-left:0;margin-right:.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mathnormal mtight">b</span></span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:.29971000000000003em"><span></span></span></span></span></span></span><span class="mspace" style="margin-right:.16666666666666666em"></span><span class="mord"><span class="mord mathnormal">e</span><span class="msupsub"><span class="vlist-t"><span class="vlist-r"><span class="vlist" style="height:.8308649999999999em"><span style="top:-2.989em;margin-right:.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mathnormal mtight">ϕ</span><span class="mopen mtight">(</span><span class="mord mathnormal mtight">s</span><span class="mpunct mtight">,</span><span class="mord mathnormal mtight">b</span><span class="mclose mtight"><span class="mclose mtight">)</span><span class="msupsub"><span class="vlist-t"><span class="vlist-r"><span class="vlist" style="height:.7740928571428571em"><span style="top:-2.786em;margin-right:.07142857142857144em"><span class="pstrut" style="height:2.5em"></span><span class="sizing reset-size3 size1 mtight"><span class="mord mathnormal mtight" style="margin-right:.13889em">T</span></span></span></span></span></span></span></span><span class="mord mathnormal mtight" style="margin-right:.02778em">θ</span></span></span></span></span></span></span></span></span></span></span><span style="top:-3.2363649999999997em"><span class="pstrut" style="height:3.0063649999999997em"></span><span class="frac-line" style="border-bottom-width:.04em"></span></span><span style="top:-3.683365em"><span class="pstrut" style="height:3.0063649999999997em"></span><span class="mord"><span class="mord"><span class="mord mathnormal">e</span><span class="msupsub"><span class="vlist-t"><span class="vlist-r"><span class="vlist" style="height:1.006365em"><span style="top:-3.063em;margin-right:.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mathnormal mtight">ϕ</span><span class="mopen mtight">(</span><span class="mord mathnormal mtight">s</span><span class="mpunct mtight">,</span><span class="mord mathnormal mtight">a</span><span class="mclose mtight"><span class="mclose mtight">)</span><span class="msupsub"><span class="vlist-t"><span class="vlist-r"><span class="vlist" style="height:.9190928571428572em"><span style="top:-2.931em;margin-right:.07142857142857144em"><span class="pstrut" style="height:2.5em"></span><span class="sizing reset-size3 size1 mtight"><span class="mord mathnormal mtight" style="margin-right:.13889em">T</span></span></span></span></span></span></span></span><span class="mord mathnormal mtight" style="margin-right:.02778em">θ</span></span></span></span></span></span></span></span></span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:1.020575em"><span></span></span></span></span></span><span class="mclose nulldelimiter"></span></span></span></span></span></span></p><p>对于<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><msub><mi mathvariant="normal">∇</mi><mi>θ</mi></msub><mi>l</mi><mi>o</mi><mi>g</mi><msub><mi>π</mi><mi>θ</mi></msub><mo stretchy="false">(</mo><mi>s</mi><mo separator="true">,</mo><mi>a</mi><mo stretchy="false">)</mo></mrow><annotation encoding="application/x-tex">∇_θlogπ_θ(s,a)</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-.25em"></span><span class="mord"><span class="mord">∇</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:.33610799999999996em"><span style="top:-2.5500000000000003em;margin-left:0;margin-right:.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight" style="margin-right:.02778em">θ</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:.15em"><span></span></span></span></span></span></span><span class="mord mathnormal" style="margin-right:.01968em">l</span><span class="mord mathnormal">o</span><span class="mord mathnormal" style="margin-right:.03588em">g</span><span class="mord"><span class="mord mathnormal" style="margin-right:.03588em">π</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:.33610799999999996em"><span style="top:-2.5500000000000003em;margin-left:-.03588em;margin-right:.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight" style="margin-right:.02778em">θ</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:.15em"><span></span></span></span></span></span></span><span class="mopen">(</span><span class="mord mathnormal">s</span><span class="mpunct">,</span><span class="mspace" style="margin-right:.16666666666666666em"></span><span class="mord mathnormal">a</span><span class="mclose">)</span></span></span></span>，我们一般称为分值函数，则通过求导很容易求出分值函数为：</p><p><span class="katex-display"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML" display="block"><semantics><mrow><msub><mi mathvariant="normal">∇</mi><mi>θ</mi></msub><mi>l</mi><mi>o</mi><mi>g</mi><msub><mi>π</mi><mi>θ</mi></msub><mo stretchy="false">(</mo><mi>s</mi><mo separator="true">,</mo><mi>a</mi><mo stretchy="false">)</mo><mo>=</mo><mi>ϕ</mi><mo stretchy="false">(</mo><mi>s</mi><mo separator="true">,</mo><mi>a</mi><mo stretchy="false">)</mo><mtext>−</mtext><msub><mi>E</mi><msub><mi>π</mi><mi>θ</mi></msub></msub><mo stretchy="false">[</mo><mi>ϕ</mi><mo stretchy="false">(</mo><mi>s</mi><mo separator="true">,</mo><mi mathvariant="normal">.</mi><mo stretchy="false">)</mo><mo stretchy="false">]</mo></mrow><annotation encoding="application/x-tex">∇_θlogπ_θ(s,a)=ϕ(s,a)−E_{π_θ}[ϕ(s,.)]</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-.25em"></span><span class="mord"><span class="mord">∇</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:.33610799999999996em"><span style="top:-2.5500000000000003em;margin-left:0;margin-right:.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight" style="margin-right:.02778em">θ</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:.15em"><span></span></span></span></span></span></span><span class="mord mathnormal" style="margin-right:.01968em">l</span><span class="mord mathnormal">o</span><span class="mord mathnormal" style="margin-right:.03588em">g</span><span class="mord"><span class="mord mathnormal" style="margin-right:.03588em">π</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:.33610799999999996em"><span style="top:-2.5500000000000003em;margin-left:-.03588em;margin-right:.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight" style="margin-right:.02778em">θ</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:.15em"><span></span></span></span></span></span></span><span class="mopen">(</span><span class="mord mathnormal">s</span><span class="mpunct">,</span><span class="mspace" style="margin-right:.16666666666666666em"></span><span class="mord mathnormal">a</span><span class="mclose">)</span><span class="mspace" style="margin-right:.2777777777777778em"></span><span class="mrel">=</span><span class="mspace" style="margin-right:.2777777777777778em"></span></span><span class="base"><span class="strut" style="height:1.00586em;vertical-align:-.25586em"></span><span class="mord mathnormal">ϕ</span><span class="mopen">(</span><span class="mord mathnormal">s</span><span class="mpunct">,</span><span class="mspace" style="margin-right:.16666666666666666em"></span><span class="mord mathnormal">a</span><span class="mclose">)</span><span class="mord">−</span><span class="mord"><span class="mord mathnormal" style="margin-right:.05764em">E</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:.15139199999999997em"><span style="top:-2.55em;margin-left:-.05764em;margin-right:.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mtight"><span class="mord mathnormal mtight" style="margin-right:.03588em">π</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:.3448em"><span style="top:-2.3487714285714287em;margin-left:-.03588em;margin-right:.07142857142857144em"><span class="pstrut" style="height:2.5em"></span><span class="sizing reset-size3 size1 mtight"><span class="mord mathnormal mtight" style="margin-right:.02778em">θ</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:.15122857142857138em"><span></span></span></span></span></span></span></span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:.25586em"><span></span></span></span></span></span></span><span class="mopen">[</span><span class="mord mathnormal">ϕ</span><span class="mopen">(</span><span class="mord mathnormal">s</span><span class="mpunct">,</span><span class="mspace" style="margin-right:.16666666666666666em"></span><span class="mord">.</span><span class="mclose">)</span><span class="mclose">]</span></span></span></span></span></p><h1 id="4-策略梯度的优化目标"><a class="anchor" href="#4-策略梯度的优化目标">#</a> 4 策略梯度的优化目标</h1><p>通常情况下目标策略有三种方式（V 是价值函数，）:</p><ul><li><p>最简单的优化目标就是初始状态收获的期望，即优化目标为：</p><p><span class="katex-display"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML" display="block"><semantics><mrow><msub><mi>J</mi><mn>1</mn></msub><mo stretchy="false">(</mo><mi>θ</mi><mo stretchy="false">)</mo><mo>=</mo><msub><mi>V</mi><msub><mi>π</mi><mi>θ</mi></msub></msub><mo stretchy="false">(</mo><msub><mi>s</mi><mn>1</mn></msub><mo stretchy="false">)</mo><mo>=</mo><msub><mi>E</mi><msub><mi>π</mi><mi>θ</mi></msub></msub><mo stretchy="false">[</mo><msub><mi>v</mi><mn>1</mn></msub><mo stretchy="false">]</mo><mo>=</mo><mi>E</mi><mo stretchy="false">(</mo><msub><mi>r</mi><mn>1</mn></msub><mo>+</mo><mi>γ</mi><msub><mi>r</mi><mn>2</mn></msub><mo>+</mo><msup><mi>γ</mi><mn>2</mn></msup><msub><mi>r</mi><mn>3</mn></msub><mo>+</mo><mi mathvariant="normal">.</mi><mi mathvariant="normal">.</mi><mi mathvariant="normal">.</mi><mi mathvariant="normal">.</mi><mi mathvariant="normal">.</mi><mi mathvariant="normal">.</mi><mi mathvariant="normal">.</mi><mo>∣</mo><msub><mi>π</mi><mi>θ</mi></msub><mo stretchy="false">)</mo></mrow><annotation encoding="application/x-tex">J_1(θ)=V_{π_θ}(s_1)=E_{π_θ}[v_1]=E(r_1+γr_2+γ^2r_3+.......∣π_θ)</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-.25em"></span><span class="mord"><span class="mord mathnormal" style="margin-right:.09618em">J</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:.30110799999999993em"><span style="top:-2.5500000000000003em;margin-left:-.09618em;margin-right:.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight">1</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:.15em"><span></span></span></span></span></span></span><span class="mopen">(</span><span class="mord mathnormal" style="margin-right:.02778em">θ</span><span class="mclose">)</span><span class="mspace" style="margin-right:.2777777777777778em"></span><span class="mrel">=</span><span class="mspace" style="margin-right:.2777777777777778em"></span></span><span class="base"><span class="strut" style="height:1.00586em;vertical-align:-.25586em"></span><span class="mord"><span class="mord mathnormal" style="margin-right:.22222em">V</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:.15139199999999997em"><span style="top:-2.55em;margin-left:-.22222em;margin-right:.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mtight"><span class="mord mathnormal mtight" style="margin-right:.03588em">π</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:.3448em"><span style="top:-2.3487714285714287em;margin-left:-.03588em;margin-right:.07142857142857144em"><span class="pstrut" style="height:2.5em"></span><span class="sizing reset-size3 size1 mtight"><span class="mord mathnormal mtight" style="margin-right:.02778em">θ</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:.15122857142857138em"><span></span></span></span></span></span></span></span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:.25586em"><span></span></span></span></span></span></span><span class="mopen">(</span><span class="mord"><span class="mord mathnormal">s</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:.30110799999999993em"><span style="top:-2.5500000000000003em;margin-left:0;margin-right:.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight">1</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:.15em"><span></span></span></span></span></span></span><span class="mclose">)</span><span class="mspace" style="margin-right:.2777777777777778em"></span><span class="mrel">=</span><span class="mspace" style="margin-right:.2777777777777778em"></span></span><span class="base"><span class="strut" style="height:1.00586em;vertical-align:-.25586em"></span><span class="mord"><span class="mord mathnormal" style="margin-right:.05764em">E</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:.15139199999999997em"><span style="top:-2.55em;margin-left:-.05764em;margin-right:.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mtight"><span class="mord mathnormal mtight" style="margin-right:.03588em">π</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:.3448em"><span style="top:-2.3487714285714287em;margin-left:-.03588em;margin-right:.07142857142857144em"><span class="pstrut" style="height:2.5em"></span><span class="sizing reset-size3 size1 mtight"><span class="mord mathnormal mtight" style="margin-right:.02778em">θ</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:.15122857142857138em"><span></span></span></span></span></span></span></span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:.25586em"><span></span></span></span></span></span></span><span class="mopen">[</span><span class="mord"><span class="mord mathnormal" style="margin-right:.03588em">v</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:.30110799999999993em"><span style="top:-2.5500000000000003em;margin-left:-.03588em;margin-right:.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight">1</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:.15em"><span></span></span></span></span></span></span><span class="mclose">]</span><span class="mspace" style="margin-right:.2777777777777778em"></span><span class="mrel">=</span><span class="mspace" style="margin-right:.2777777777777778em"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-.25em"></span><span class="mord mathnormal" style="margin-right:.05764em">E</span><span class="mopen">(</span><span class="mord"><span class="mord mathnormal" style="margin-right:.02778em">r</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:.30110799999999993em"><span style="top:-2.5500000000000003em;margin-left:-.02778em;margin-right:.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight">1</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:.15em"><span></span></span></span></span></span></span><span class="mspace" style="margin-right:.2222222222222222em"></span><span class="mbin">+</span><span class="mspace" style="margin-right:.2222222222222222em"></span></span><span class="base"><span class="strut" style="height:.7777700000000001em;vertical-align:-.19444em"></span><span class="mord mathnormal" style="margin-right:.05556em">γ</span><span class="mord"><span class="mord mathnormal" style="margin-right:.02778em">r</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:.30110799999999993em"><span style="top:-2.5500000000000003em;margin-left:-.02778em;margin-right:.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight">2</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:.15em"><span></span></span></span></span></span></span><span class="mspace" style="margin-right:.2222222222222222em"></span><span class="mbin">+</span><span class="mspace" style="margin-right:.2222222222222222em"></span></span><span class="base"><span class="strut" style="height:1.0585479999999998em;vertical-align:-.19444em"></span><span class="mord"><span class="mord mathnormal" style="margin-right:.05556em">γ</span><span class="msupsub"><span class="vlist-t"><span class="vlist-r"><span class="vlist" style="height:.8641079999999999em"><span style="top:-3.113em;margin-right:.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight">2</span></span></span></span></span></span></span></span><span class="mord"><span class="mord mathnormal" style="margin-right:.02778em">r</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:.30110799999999993em"><span style="top:-2.5500000000000003em;margin-left:-.02778em;margin-right:.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight">3</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:.15em"><span></span></span></span></span></span></span><span class="mspace" style="margin-right:.2222222222222222em"></span><span class="mbin">+</span><span class="mspace" style="margin-right:.2222222222222222em"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-.25em"></span><span class="mord">.</span><span class="mord">.</span><span class="mord">.</span><span class="mord">.</span><span class="mord">.</span><span class="mord">.</span><span class="mord">.</span><span class="mspace" style="margin-right:.2777777777777778em"></span><span class="mrel">∣</span><span class="mspace" style="margin-right:.2777777777777778em"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-.25em"></span><span class="mord"><span class="mord mathnormal" style="margin-right:.03588em">π</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:.33610799999999996em"><span style="top:-2.5500000000000003em;margin-left:-.03588em;margin-right:.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight" style="margin-right:.02778em">θ</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:.15em"><span></span></span></span></span></span></span><span class="mclose">)</span></span></span></span></span></p></li><li><p>但是有的问题是没有明确的初始状态的，那么我们的优化目标可以定义平均价值，即：</p><p><span class="katex-display"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML" display="block"><semantics><mrow><msub><mi>J</mi><mrow><mi>a</mi><mi>v</mi><mi>V</mi></mrow></msub><mo stretchy="false">(</mo><mi>θ</mi><mo stretchy="false">)</mo><mo>=</mo><munder><mo>∑</mo><mi>s</mi></munder><msub><mi>d</mi><msub><mi>π</mi><mi>θ</mi></msub></msub><mo stretchy="false">(</mo><mi>s</mi><mo stretchy="false">)</mo><msub><mi>V</mi><msub><mi>π</mi><mi>θ</mi></msub></msub><mo stretchy="false">(</mo><mi>s</mi><mo stretchy="false">)</mo></mrow><annotation encoding="application/x-tex">J_{avV}(θ)=\sum_{s}d_{π_θ}(s)V_{π_θ}(s)</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-.25em"></span><span class="mord"><span class="mord mathnormal" style="margin-right:.09618em">J</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:.32833099999999993em"><span style="top:-2.5500000000000003em;margin-left:-.09618em;margin-right:.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mathnormal mtight">a</span><span class="mord mathnormal mtight" style="margin-right:.03588em">v</span><span class="mord mathnormal mtight" style="margin-right:.22222em">V</span></span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:.15em"><span></span></span></span></span></span></span><span class="mopen">(</span><span class="mord mathnormal" style="margin-right:.02778em">θ</span><span class="mclose">)</span><span class="mspace" style="margin-right:.2777777777777778em"></span><span class="mrel">=</span><span class="mspace" style="margin-right:.2777777777777778em"></span></span><span class="base"><span class="strut" style="height:2.3000100000000003em;vertical-align:-1.250005em"></span><span class="mop op-limits"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:1.050005em"><span style="top:-1.8999949999999999em;margin-left:0"><span class="pstrut" style="height:3.05em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mathnormal mtight">s</span></span></span></span><span style="top:-3.0500049999999996em"><span class="pstrut" style="height:3.05em"></span><span><span class="mop op-symbol large-op">∑</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:1.250005em"><span></span></span></span></span></span><span class="mspace" style="margin-right:.16666666666666666em"></span><span class="mord"><span class="mord mathnormal">d</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:.15139199999999997em"><span style="top:-2.55em;margin-left:0;margin-right:.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mtight"><span class="mord mathnormal mtight" style="margin-right:.03588em">π</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:.3448em"><span style="top:-2.3487714285714287em;margin-left:-.03588em;margin-right:.07142857142857144em"><span class="pstrut" style="height:2.5em"></span><span class="sizing reset-size3 size1 mtight"><span class="mord mathnormal mtight" style="margin-right:.02778em">θ</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:.15122857142857138em"><span></span></span></span></span></span></span></span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:.25586em"><span></span></span></span></span></span></span><span class="mopen">(</span><span class="mord mathnormal">s</span><span class="mclose">)</span><span class="mord"><span class="mord mathnormal" style="margin-right:.22222em">V</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:.15139199999999997em"><span style="top:-2.55em;margin-left:-.22222em;margin-right:.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mtight"><span class="mord mathnormal mtight" style="margin-right:.03588em">π</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:.3448em"><span style="top:-2.3487714285714287em;margin-left:-.03588em;margin-right:.07142857142857144em"><span class="pstrut" style="height:2.5em"></span><span class="sizing reset-size3 size1 mtight"><span class="mord mathnormal mtight" style="margin-right:.02778em">θ</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:.15122857142857138em"><span></span></span></span></span></span></span></span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:.25586em"><span></span></span></span></span></span></span><span class="mopen">(</span><span class="mord mathnormal">s</span><span class="mclose">)</span></span></span></span></span></p><p>其中，<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><msub><mi>d</mi><msub><mi>π</mi><mi>θ</mi></msub></msub><mo stretchy="false">(</mo><mi>s</mi><mo stretchy="false">)</mo></mrow><annotation encoding="application/x-tex">d_{π_θ}(s)</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1.00586em;vertical-align:-.25586em"></span><span class="mord"><span class="mord mathnormal">d</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:.15139199999999997em"><span style="top:-2.55em;margin-left:0;margin-right:.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mtight"><span class="mord mathnormal mtight" style="margin-right:.03588em">π</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:.3448em"><span style="top:-2.3487714285714287em;margin-left:-.03588em;margin-right:.07142857142857144em"><span class="pstrut" style="height:2.5em"></span><span class="sizing reset-size3 size1 mtight"><span class="mord mathnormal mtight" style="margin-right:.02778em">θ</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:.15122857142857138em"><span></span></span></span></span></span></span></span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:.25586em"><span></span></span></span></span></span></span><span class="mopen">(</span><span class="mord mathnormal">s</span><span class="mclose">)</span></span></span></span> 是基于策略<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><msub><mi>π</mi><mi>θ</mi></msub></mrow><annotation encoding="application/x-tex">π_θ</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:.58056em;vertical-align:-.15em"></span><span class="mord"><span class="mord mathnormal" style="margin-right:.03588em">π</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:.33610799999999996em"><span style="top:-2.5500000000000003em;margin-left:-.03588em;margin-right:.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight" style="margin-right:.02778em">θ</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:.15em"><span></span></span></span></span></span></span></span></span></span> 生成的马尔科夫链关于状态的静态分布。</p></li><li><p>还有就是定义为每一时间步的平均奖励，即：</p><p><span class="katex-display"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML" display="block"><semantics><mrow><msub><mi>J</mi><mrow><mi>a</mi><mi>v</mi><mi>R</mi></mrow></msub><mo stretchy="false">(</mo><mi>θ</mi><mo stretchy="false">)</mo><mo>=</mo><mo>=</mo><munder><mo>∑</mo><mi>s</mi></munder><msub><mi>d</mi><msub><mi>π</mi><mi>θ</mi></msub></msub><mo stretchy="false">(</mo><mi>s</mi><mo stretchy="false">)</mo><munder><mo>∑</mo><mi>a</mi></munder><msub><mi>π</mi><mi>θ</mi></msub><mo stretchy="false">(</mo><mi>s</mi><mo separator="true">,</mo><mi>a</mi><mo stretchy="false">)</mo><msubsup><mi>R</mi><mi>s</mi><mi>a</mi></msubsup></mrow><annotation encoding="application/x-tex">J_{avR}(θ)==\sum_sd_{π_θ}(s)\sum_aπ_θ(s,a)R^a_s</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-.25em"></span><span class="mord"><span class="mord mathnormal" style="margin-right:.09618em">J</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:.32833099999999993em"><span style="top:-2.5500000000000003em;margin-left:-.09618em;margin-right:.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mathnormal mtight">a</span><span class="mord mathnormal mtight" style="margin-right:.03588em">v</span><span class="mord mathnormal mtight" style="margin-right:.00773em">R</span></span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:.15em"><span></span></span></span></span></span></span><span class="mopen">(</span><span class="mord mathnormal" style="margin-right:.02778em">θ</span><span class="mclose">)</span><span class="mspace" style="margin-right:.2777777777777778em"></span><span class="mrel">=</span></span><span class="base"><span class="strut" style="height:.36687em;vertical-align:0"></span><span class="mrel">=</span><span class="mspace" style="margin-right:.2777777777777778em"></span></span><span class="base"><span class="strut" style="height:2.3000100000000003em;vertical-align:-1.250005em"></span><span class="mop op-limits"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:1.050005em"><span style="top:-1.8999949999999999em;margin-left:0"><span class="pstrut" style="height:3.05em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">s</span></span></span><span style="top:-3.0500049999999996em"><span class="pstrut" style="height:3.05em"></span><span><span class="mop op-symbol large-op">∑</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:1.250005em"><span></span></span></span></span></span><span class="mspace" style="margin-right:.16666666666666666em"></span><span class="mord"><span class="mord mathnormal">d</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:.15139199999999997em"><span style="top:-2.55em;margin-left:0;margin-right:.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mtight"><span class="mord mathnormal mtight" style="margin-right:.03588em">π</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:.3448em"><span style="top:-2.3487714285714287em;margin-left:-.03588em;margin-right:.07142857142857144em"><span class="pstrut" style="height:2.5em"></span><span class="sizing reset-size3 size1 mtight"><span class="mord mathnormal mtight" style="margin-right:.02778em">θ</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:.15122857142857138em"><span></span></span></span></span></span></span></span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:.25586em"><span></span></span></span></span></span></span><span class="mopen">(</span><span class="mord mathnormal">s</span><span class="mclose">)</span><span class="mspace" style="margin-right:.16666666666666666em"></span><span class="mop op-limits"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:1.050005em"><span style="top:-1.8999949999999999em;margin-left:0"><span class="pstrut" style="height:3.05em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">a</span></span></span><span style="top:-3.0500049999999996em"><span class="pstrut" style="height:3.05em"></span><span><span class="mop op-symbol large-op">∑</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:1.250005em"><span></span></span></span></span></span><span class="mspace" style="margin-right:.16666666666666666em"></span><span class="mord"><span class="mord mathnormal" style="margin-right:.03588em">π</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:.33610799999999996em"><span style="top:-2.5500000000000003em;margin-left:-.03588em;margin-right:.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight" style="margin-right:.02778em">θ</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:.15em"><span></span></span></span></span></span></span><span class="mopen">(</span><span class="mord mathnormal">s</span><span class="mpunct">,</span><span class="mspace" style="margin-right:.16666666666666666em"></span><span class="mord mathnormal">a</span><span class="mclose">)</span><span class="mord"><span class="mord mathnormal" style="margin-right:.00773em">R</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:.7143919999999999em"><span style="top:-2.4530000000000003em;margin-left:-.00773em;margin-right:.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">s</span></span></span><span style="top:-3.113em;margin-right:.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">a</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:.247em"><span></span></span></span></span></span></span></span></span></span></span></p></li></ul><p>无论我们是采用<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><msub><mi>J</mi><mn>1</mn></msub><mo separator="true">,</mo><msub><mi>J</mi><mrow><mi>a</mi><mi>v</mi><mi>V</mi></mrow></msub></mrow><annotation encoding="application/x-tex">J_1,J_{avV}</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:.8777699999999999em;vertical-align:-.19444em"></span><span class="mord"><span class="mord mathnormal" style="margin-right:.09618em">J</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:.30110799999999993em"><span style="top:-2.5500000000000003em;margin-left:-.09618em;margin-right:.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight">1</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:.15em"><span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:.16666666666666666em"></span><span class="mord"><span class="mord mathnormal" style="margin-right:.09618em">J</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:.32833099999999993em"><span style="top:-2.5500000000000003em;margin-left:-.09618em;margin-right:.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mathnormal mtight">a</span><span class="mord mathnormal mtight" style="margin-right:.03588em">v</span><span class="mord mathnormal mtight" style="margin-right:.22222em">V</span></span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:.15em"><span></span></span></span></span></span></span></span></span></span> 还是<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><msub><mi>J</mi><mrow><mi>a</mi><mi>v</mi><mi>R</mi></mrow></msub></mrow><annotation encoding="application/x-tex">J_{avR}</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:.83333em;vertical-align:-.15em"></span><span class="mord"><span class="mord mathnormal" style="margin-right:.09618em">J</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:.32833099999999993em"><span style="top:-2.5500000000000003em;margin-left:-.09618em;margin-right:.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mathnormal mtight">a</span><span class="mord mathnormal mtight" style="margin-right:.03588em">v</span><span class="mord mathnormal mtight" style="margin-right:.00773em">R</span></span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:.15em"><span></span></span></span></span></span></span></span></span></span> 来表示优化目标，最终对<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>θ</mi></mrow><annotation encoding="application/x-tex">θ</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:.69444em;vertical-align:0"></span><span class="mord mathnormal" style="margin-right:.02778em">θ</span></span></span></span> 求导的梯度都可以表示为：</p><p><span class="katex-display"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML" display="block"><semantics><mrow><msub><mi mathvariant="normal">∇</mi><mi>θ</mi></msub><mi>J</mi><mo stretchy="false">(</mo><mi>θ</mi><mo stretchy="false">)</mo><mo>=</mo><msub><mi>E</mi><msub><mi>π</mi><mi>θ</mi></msub></msub><mo stretchy="false">[</mo><msub><mi mathvariant="normal">∇</mi><mi>θ</mi></msub><mi>l</mi><mi>o</mi><mi>g</mi><msub><mi>π</mi><mi>θ</mi></msub><mo stretchy="false">(</mo><mi>s</mi><mo separator="true">,</mo><mi>a</mi><mo stretchy="false">)</mo><msub><mi>Q</mi><mi>π</mi></msub><mo stretchy="false">(</mo><mi>s</mi><mo separator="true">,</mo><mi>a</mi><mo stretchy="false">)</mo><mo stretchy="false">]</mo></mrow><annotation encoding="application/x-tex">∇_θJ(θ)=E_{π_θ}[∇_θlogπ_θ(s,a)Q_π(s,a)]</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-.25em"></span><span class="mord"><span class="mord">∇</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:.33610799999999996em"><span style="top:-2.5500000000000003em;margin-left:0;margin-right:.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight" style="margin-right:.02778em">θ</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:.15em"><span></span></span></span></span></span></span><span class="mord mathnormal" style="margin-right:.09618em">J</span><span class="mopen">(</span><span class="mord mathnormal" style="margin-right:.02778em">θ</span><span class="mclose">)</span><span class="mspace" style="margin-right:.2777777777777778em"></span><span class="mrel">=</span><span class="mspace" style="margin-right:.2777777777777778em"></span></span><span class="base"><span class="strut" style="height:1.00586em;vertical-align:-.25586em"></span><span class="mord"><span class="mord mathnormal" style="margin-right:.05764em">E</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:.15139199999999997em"><span style="top:-2.55em;margin-left:-.05764em;margin-right:.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mtight"><span class="mord mathnormal mtight" style="margin-right:.03588em">π</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:.3448em"><span style="top:-2.3487714285714287em;margin-left:-.03588em;margin-right:.07142857142857144em"><span class="pstrut" style="height:2.5em"></span><span class="sizing reset-size3 size1 mtight"><span class="mord mathnormal mtight" style="margin-right:.02778em">θ</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:.15122857142857138em"><span></span></span></span></span></span></span></span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:.25586em"><span></span></span></span></span></span></span><span class="mopen">[</span><span class="mord"><span class="mord">∇</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:.33610799999999996em"><span style="top:-2.5500000000000003em;margin-left:0;margin-right:.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight" style="margin-right:.02778em">θ</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:.15em"><span></span></span></span></span></span></span><span class="mord mathnormal" style="margin-right:.01968em">l</span><span class="mord mathnormal">o</span><span class="mord mathnormal" style="margin-right:.03588em">g</span><span class="mord"><span class="mord mathnormal" style="margin-right:.03588em">π</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:.33610799999999996em"><span style="top:-2.5500000000000003em;margin-left:-.03588em;margin-right:.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight" style="margin-right:.02778em">θ</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:.15em"><span></span></span></span></span></span></span><span class="mopen">(</span><span class="mord mathnormal">s</span><span class="mpunct">,</span><span class="mspace" style="margin-right:.16666666666666666em"></span><span class="mord mathnormal">a</span><span class="mclose">)</span><span class="mord"><span class="mord mathnormal">Q</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:.151392em"><span style="top:-2.5500000000000003em;margin-left:0;margin-right:.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight" style="margin-right:.03588em">π</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:.15em"><span></span></span></span></span></span></span><span class="mopen">(</span><span class="mord mathnormal">s</span><span class="mpunct">,</span><span class="mspace" style="margin-right:.16666666666666666em"></span><span class="mord mathnormal">a</span><span class="mclose">)</span><span class="mclose">]</span></span></span></span></span></p><p>详细推导可以参考原论文：<span class="exturl" data-url="aHR0cHM6Ly9ob21lcy5jcy53YXNoaW5ndG9uLmVkdS9+dG9kb3Jvdi9jb3Vyc2VzL2FtYXRoNTc5L3JlYWRpbmcvUG9saWN5R3JhZGllbnQucGRm">论文</span></p><h1 id="5-代码实现"><a class="anchor" href="#5-代码实现">#</a> 5 代码实现</h1><p>这里使用了 OpenAI Gym 中的 CartPole-v0 游戏来作为我们算法应用。CartPole-v0 游戏的介绍参见<span class="exturl" data-url="aHR0cHM6Ly9naXRodWIuY29tL29wZW5haS9neW0vd2lraS9DYXJ0UG9sZS12MA==">这里</span>。它比较简单，基本要求就是控制下面的 cart 移动使连接在上面的 pole 保持垂直不倒。这个任务只有两个离散动作，要么向左用力，要么向右用力。而 state 状态就是这个 cart 的位置和速度， pole 的角度和角速度，4 维的特征。坚持到 200 分的奖励则为过关。</p><figure class="highlight python"><figcaption data-lang="python"></figcaption><table><tr><td data-num="1"></td><td><pre><span class="token keyword">import</span> gym</pre></td></tr><tr><td data-num="2"></td><td><pre><span class="token keyword">import</span> tensorflow <span class="token keyword">as</span> tf</pre></td></tr><tr><td data-num="3"></td><td><pre><span class="token keyword">import</span> numpy <span class="token keyword">as</span> np</pre></td></tr><tr><td data-num="4"></td><td><pre><span class="token keyword">import</span> random</pre></td></tr><tr><td data-num="5"></td><td><pre><span class="token keyword">from</span> collections <span class="token keyword">import</span> deque</pre></td></tr><tr><td data-num="6"></td><td><pre></pre></td></tr><tr><td data-num="7"></td><td><pre><span class="token comment"># Hyper Parameters</span></pre></td></tr><tr><td data-num="8"></td><td><pre>GAMMA <span class="token operator">=</span> <span class="token number">0.95</span> <span class="token comment"># discount factor</span></pre></td></tr><tr><td data-num="9"></td><td><pre>LEARNING_RATE<span class="token operator">=</span><span class="token number">0.01</span></pre></td></tr><tr><td data-num="10"></td><td><pre></pre></td></tr><tr><td data-num="11"></td><td><pre><span class="token keyword">class</span> <span class="token class-name">Policy_Gradient</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">:</span></pre></td></tr><tr><td data-num="12"></td><td><pre>    <span class="token keyword">def</span> <span class="token function">__init__</span><span class="token punctuation">(</span>self<span class="token punctuation">,</span> env<span class="token punctuation">)</span><span class="token punctuation">:</span></pre></td></tr><tr><td data-num="13"></td><td><pre>        <span class="token comment"># init some parameters</span></pre></td></tr><tr><td data-num="14"></td><td><pre>        self<span class="token punctuation">.</span>time_step <span class="token operator">=</span> <span class="token number">0</span></pre></td></tr><tr><td data-num="15"></td><td><pre>        self<span class="token punctuation">.</span>state_dim <span class="token operator">=</span> env<span class="token punctuation">.</span>observation_space<span class="token punctuation">.</span>shape<span class="token punctuation">[</span><span class="token number">0</span><span class="token punctuation">]</span></pre></td></tr><tr><td data-num="16"></td><td><pre>        self<span class="token punctuation">.</span>action_dim <span class="token operator">=</span> env<span class="token punctuation">.</span>action_space<span class="token punctuation">.</span>n</pre></td></tr><tr><td data-num="17"></td><td><pre>        self<span class="token punctuation">.</span>ep_obs<span class="token punctuation">,</span> self<span class="token punctuation">.</span>ep_as<span class="token punctuation">,</span> self<span class="token punctuation">.</span>ep_rs <span class="token operator">=</span> <span class="token punctuation">[</span><span class="token punctuation">]</span><span class="token punctuation">,</span> <span class="token punctuation">[</span><span class="token punctuation">]</span><span class="token punctuation">,</span> <span class="token punctuation">[</span><span class="token punctuation">]</span></pre></td></tr><tr><td data-num="18"></td><td><pre>        self<span class="token punctuation">.</span>create_softmax_network<span class="token punctuation">(</span><span class="token punctuation">)</span></pre></td></tr><tr><td data-num="19"></td><td><pre></pre></td></tr><tr><td data-num="20"></td><td><pre>        <span class="token comment"># Init session</span></pre></td></tr><tr><td data-num="21"></td><td><pre>        self<span class="token punctuation">.</span>session <span class="token operator">=</span> tf<span class="token punctuation">.</span>InteractiveSession<span class="token punctuation">(</span><span class="token punctuation">)</span></pre></td></tr><tr><td data-num="22"></td><td><pre>        self<span class="token punctuation">.</span>session<span class="token punctuation">.</span>run<span class="token punctuation">(</span>tf<span class="token punctuation">.</span>global_variables_initializer<span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">)</span></pre></td></tr><tr><td data-num="23"></td><td><pre></pre></td></tr><tr><td data-num="24"></td><td><pre>    <span class="token keyword">def</span> <span class="token function">create_softmax_network</span><span class="token punctuation">(</span>self<span class="token punctuation">)</span><span class="token punctuation">:</span></pre></td></tr><tr><td data-num="25"></td><td><pre>        <span class="token comment"># network weights</span></pre></td></tr><tr><td data-num="26"></td><td><pre>        W1 <span class="token operator">=</span> self<span class="token punctuation">.</span>weight_variable<span class="token punctuation">(</span><span class="token punctuation">[</span>self<span class="token punctuation">.</span>state_dim<span class="token punctuation">,</span> <span class="token number">20</span><span class="token punctuation">]</span><span class="token punctuation">)</span></pre></td></tr><tr><td data-num="27"></td><td><pre>        b1 <span class="token operator">=</span> self<span class="token punctuation">.</span>bias_variable<span class="token punctuation">(</span><span class="token punctuation">[</span><span class="token number">20</span><span class="token punctuation">]</span><span class="token punctuation">)</span></pre></td></tr><tr><td data-num="28"></td><td><pre>        W2 <span class="token operator">=</span> self<span class="token punctuation">.</span>weight_variable<span class="token punctuation">(</span><span class="token punctuation">[</span><span class="token number">20</span><span class="token punctuation">,</span> self<span class="token punctuation">.</span>action_dim<span class="token punctuation">]</span><span class="token punctuation">)</span></pre></td></tr><tr><td data-num="29"></td><td><pre>        b2 <span class="token operator">=</span> self<span class="token punctuation">.</span>bias_variable<span class="token punctuation">(</span><span class="token punctuation">[</span>self<span class="token punctuation">.</span>action_dim<span class="token punctuation">]</span><span class="token punctuation">)</span></pre></td></tr><tr><td data-num="30"></td><td><pre>        <span class="token comment"># input layer</span></pre></td></tr><tr><td data-num="31"></td><td><pre>        self<span class="token punctuation">.</span>state_input <span class="token operator">=</span> tf<span class="token punctuation">.</span>placeholder<span class="token punctuation">(</span><span class="token string">"float"</span><span class="token punctuation">,</span> <span class="token punctuation">[</span><span class="token boolean">None</span><span class="token punctuation">,</span> self<span class="token punctuation">.</span>state_dim<span class="token punctuation">]</span><span class="token punctuation">)</span></pre></td></tr><tr><td data-num="32"></td><td><pre>        self<span class="token punctuation">.</span>tf_acts <span class="token operator">=</span> tf<span class="token punctuation">.</span>placeholder<span class="token punctuation">(</span>tf<span class="token punctuation">.</span>int32<span class="token punctuation">,</span> <span class="token punctuation">[</span><span class="token boolean">None</span><span class="token punctuation">,</span> <span class="token punctuation">]</span><span class="token punctuation">,</span> name<span class="token operator">=</span><span class="token string">"actions_num"</span><span class="token punctuation">)</span></pre></td></tr><tr><td data-num="33"></td><td><pre>        self<span class="token punctuation">.</span>tf_vt <span class="token operator">=</span> tf<span class="token punctuation">.</span>placeholder<span class="token punctuation">(</span>tf<span class="token punctuation">.</span>float32<span class="token punctuation">,</span> <span class="token punctuation">[</span><span class="token boolean">None</span><span class="token punctuation">,</span> <span class="token punctuation">]</span><span class="token punctuation">,</span> name<span class="token operator">=</span><span class="token string">"actions_value"</span><span class="token punctuation">)</span></pre></td></tr><tr><td data-num="34"></td><td><pre>        <span class="token comment"># hidden layers</span></pre></td></tr><tr><td data-num="35"></td><td><pre>        h_layer <span class="token operator">=</span> tf<span class="token punctuation">.</span>nn<span class="token punctuation">.</span>relu<span class="token punctuation">(</span>tf<span class="token punctuation">.</span>matmul<span class="token punctuation">(</span>self<span class="token punctuation">.</span>state_input<span class="token punctuation">,</span> W1<span class="token punctuation">)</span> <span class="token operator">+</span> b1<span class="token punctuation">)</span></pre></td></tr><tr><td data-num="36"></td><td><pre>        <span class="token comment"># softmax layer</span></pre></td></tr><tr><td data-num="37"></td><td><pre>        self<span class="token punctuation">.</span>softmax_input <span class="token operator">=</span> tf<span class="token punctuation">.</span>matmul<span class="token punctuation">(</span>h_layer<span class="token punctuation">,</span> W2<span class="token punctuation">)</span> <span class="token operator">+</span> b2</pre></td></tr><tr><td data-num="38"></td><td><pre>        <span class="token comment">#softmax output</span></pre></td></tr><tr><td data-num="39"></td><td><pre>        self<span class="token punctuation">.</span>all_act_prob <span class="token operator">=</span> tf<span class="token punctuation">.</span>nn<span class="token punctuation">.</span>softmax<span class="token punctuation">(</span>self<span class="token punctuation">.</span>softmax_input<span class="token punctuation">,</span> name<span class="token operator">=</span><span class="token string">'act_prob'</span><span class="token punctuation">)</span></pre></td></tr><tr><td data-num="40"></td><td><pre>        self<span class="token punctuation">.</span>neg_log_prob <span class="token operator">=</span> tf<span class="token punctuation">.</span>nn<span class="token punctuation">.</span>sparse_softmax_cross_entropy_with_logits<span class="token punctuation">(</span>logits<span class="token operator">=</span>self<span class="token punctuation">.</span>softmax_input<span class="token punctuation">,</span></pre></td></tr><tr><td data-num="41"></td><td><pre>                                                                      labels<span class="token operator">=</span>self<span class="token punctuation">.</span>tf_acts<span class="token punctuation">)</span></pre></td></tr><tr><td data-num="42"></td><td><pre>        self<span class="token punctuation">.</span>loss <span class="token operator">=</span> tf<span class="token punctuation">.</span>reduce_mean<span class="token punctuation">(</span>self<span class="token punctuation">.</span>neg_log_prob <span class="token operator">*</span> self<span class="token punctuation">.</span>tf_vt<span class="token punctuation">)</span>  <span class="token comment"># reward guided loss</span></pre></td></tr><tr><td data-num="43"></td><td><pre></pre></td></tr><tr><td data-num="44"></td><td><pre>        self<span class="token punctuation">.</span>train_op <span class="token operator">=</span> tf<span class="token punctuation">.</span>train<span class="token punctuation">.</span>AdamOptimizer<span class="token punctuation">(</span>LEARNING_RATE<span class="token punctuation">)</span><span class="token punctuation">.</span>minimize<span class="token punctuation">(</span>self<span class="token punctuation">.</span>loss<span class="token punctuation">)</span></pre></td></tr><tr><td data-num="45"></td><td><pre></pre></td></tr><tr><td data-num="46"></td><td><pre>    <span class="token keyword">def</span> <span class="token function">weight_variable</span><span class="token punctuation">(</span>self<span class="token punctuation">,</span> shape<span class="token punctuation">)</span><span class="token punctuation">:</span></pre></td></tr><tr><td data-num="47"></td><td><pre>        initial <span class="token operator">=</span> tf<span class="token punctuation">.</span>truncated_normal<span class="token punctuation">(</span>shape<span class="token punctuation">)</span></pre></td></tr><tr><td data-num="48"></td><td><pre>        <span class="token keyword">return</span> tf<span class="token punctuation">.</span>Variable<span class="token punctuation">(</span>initial<span class="token punctuation">)</span></pre></td></tr><tr><td data-num="49"></td><td><pre></pre></td></tr><tr><td data-num="50"></td><td><pre>    <span class="token keyword">def</span> <span class="token function">bias_variable</span><span class="token punctuation">(</span>self<span class="token punctuation">,</span> shape<span class="token punctuation">)</span><span class="token punctuation">:</span></pre></td></tr><tr><td data-num="51"></td><td><pre>        initial <span class="token operator">=</span> tf<span class="token punctuation">.</span>constant<span class="token punctuation">(</span><span class="token number">0.01</span><span class="token punctuation">,</span> shape<span class="token operator">=</span>shape<span class="token punctuation">)</span></pre></td></tr><tr><td data-num="52"></td><td><pre>        <span class="token keyword">return</span> tf<span class="token punctuation">.</span>Variable<span class="token punctuation">(</span>initial<span class="token punctuation">)</span></pre></td></tr><tr><td data-num="53"></td><td><pre></pre></td></tr><tr><td data-num="54"></td><td><pre>    <span class="token keyword">def</span> <span class="token function">choose_action</span><span class="token punctuation">(</span>self<span class="token punctuation">,</span> observation<span class="token punctuation">)</span><span class="token punctuation">:</span></pre></td></tr><tr><td data-num="55"></td><td><pre>        prob_weights <span class="token operator">=</span> self<span class="token punctuation">.</span>session<span class="token punctuation">.</span>run<span class="token punctuation">(</span>self<span class="token punctuation">.</span>all_act_prob<span class="token punctuation">,</span> feed_dict<span class="token operator">=</span><span class="token punctuation">&#123;</span>self<span class="token punctuation">.</span>state_input<span class="token punctuation">:</span> observation<span class="token punctuation">[</span>np<span class="token punctuation">.</span>newaxis<span class="token punctuation">,</span> <span class="token punctuation">:</span><span class="token punctuation">]</span><span class="token punctuation">&#125;</span><span class="token punctuation">)</span></pre></td></tr><tr><td data-num="56"></td><td><pre>        action <span class="token operator">=</span> np<span class="token punctuation">.</span>random<span class="token punctuation">.</span>choice<span class="token punctuation">(</span><span class="token builtin">range</span><span class="token punctuation">(</span>prob_weights<span class="token punctuation">.</span>shape<span class="token punctuation">[</span><span class="token number">1</span><span class="token punctuation">]</span><span class="token punctuation">)</span><span class="token punctuation">,</span> p<span class="token operator">=</span>prob_weights<span class="token punctuation">.</span>ravel<span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">)</span>  <span class="token comment"># select action w.r.t the actions prob</span></pre></td></tr><tr><td data-num="57"></td><td><pre>        <span class="token keyword">return</span> action</pre></td></tr><tr><td data-num="58"></td><td><pre></pre></td></tr><tr><td data-num="59"></td><td><pre>    <span class="token keyword">def</span> <span class="token function">store_transition</span><span class="token punctuation">(</span>self<span class="token punctuation">,</span> s<span class="token punctuation">,</span> a<span class="token punctuation">,</span> r<span class="token punctuation">)</span><span class="token punctuation">:</span></pre></td></tr><tr><td data-num="60"></td><td><pre>        self<span class="token punctuation">.</span>ep_obs<span class="token punctuation">.</span>append<span class="token punctuation">(</span>s<span class="token punctuation">)</span></pre></td></tr><tr><td data-num="61"></td><td><pre>        self<span class="token punctuation">.</span>ep_as<span class="token punctuation">.</span>append<span class="token punctuation">(</span>a<span class="token punctuation">)</span></pre></td></tr><tr><td data-num="62"></td><td><pre>        self<span class="token punctuation">.</span>ep_rs<span class="token punctuation">.</span>append<span class="token punctuation">(</span>r<span class="token punctuation">)</span></pre></td></tr><tr><td data-num="63"></td><td><pre></pre></td></tr><tr><td data-num="64"></td><td><pre>    <span class="token keyword">def</span> <span class="token function">learn</span><span class="token punctuation">(</span>self<span class="token punctuation">)</span><span class="token punctuation">:</span></pre></td></tr><tr><td data-num="65"></td><td><pre></pre></td></tr><tr><td data-num="66"></td><td><pre>        discounted_ep_rs <span class="token operator">=</span> np<span class="token punctuation">.</span>zeros_like<span class="token punctuation">(</span>self<span class="token punctuation">.</span>ep_rs<span class="token punctuation">)</span></pre></td></tr><tr><td data-num="67"></td><td><pre>        running_add <span class="token operator">=</span> <span class="token number">0</span></pre></td></tr><tr><td data-num="68"></td><td><pre>        <span class="token keyword">for</span> t <span class="token keyword">in</span> <span class="token builtin">reversed</span><span class="token punctuation">(</span><span class="token builtin">range</span><span class="token punctuation">(</span><span class="token number">0</span><span class="token punctuation">,</span> <span class="token builtin">len</span><span class="token punctuation">(</span>self<span class="token punctuation">.</span>ep_rs<span class="token punctuation">)</span><span class="token punctuation">)</span><span class="token punctuation">)</span><span class="token punctuation">:</span></pre></td></tr><tr><td data-num="69"></td><td><pre>            running_add <span class="token operator">=</span> running_add <span class="token operator">*</span> GAMMA <span class="token operator">+</span> self<span class="token punctuation">.</span>ep_rs<span class="token punctuation">[</span>t<span class="token punctuation">]</span></pre></td></tr><tr><td data-num="70"></td><td><pre>            discounted_ep_rs<span class="token punctuation">[</span>t<span class="token punctuation">]</span> <span class="token operator">=</span> running_add</pre></td></tr><tr><td data-num="71"></td><td><pre></pre></td></tr><tr><td data-num="72"></td><td><pre>        discounted_ep_rs <span class="token operator">-=</span> np<span class="token punctuation">.</span>mean<span class="token punctuation">(</span>discounted_ep_rs<span class="token punctuation">)</span></pre></td></tr><tr><td data-num="73"></td><td><pre>        discounted_ep_rs <span class="token operator">/=</span> np<span class="token punctuation">.</span>std<span class="token punctuation">(</span>discounted_ep_rs<span class="token punctuation">)</span></pre></td></tr><tr><td data-num="74"></td><td><pre></pre></td></tr><tr><td data-num="75"></td><td><pre>        <span class="token comment"># train on episode</span></pre></td></tr><tr><td data-num="76"></td><td><pre>        self<span class="token punctuation">.</span>session<span class="token punctuation">.</span>run<span class="token punctuation">(</span>self<span class="token punctuation">.</span>train_op<span class="token punctuation">,</span> feed_dict<span class="token operator">=</span><span class="token punctuation">&#123;</span></pre></td></tr><tr><td data-num="77"></td><td><pre>             self<span class="token punctuation">.</span>state_input<span class="token punctuation">:</span> np<span class="token punctuation">.</span>vstack<span class="token punctuation">(</span>self<span class="token punctuation">.</span>ep_obs<span class="token punctuation">)</span><span class="token punctuation">,</span></pre></td></tr><tr><td data-num="78"></td><td><pre>             self<span class="token punctuation">.</span>tf_acts<span class="token punctuation">:</span> np<span class="token punctuation">.</span>array<span class="token punctuation">(</span>self<span class="token punctuation">.</span>ep_as<span class="token punctuation">)</span><span class="token punctuation">,</span></pre></td></tr><tr><td data-num="79"></td><td><pre>             self<span class="token punctuation">.</span>tf_vt<span class="token punctuation">:</span> discounted_ep_rs<span class="token punctuation">,</span></pre></td></tr><tr><td data-num="80"></td><td><pre>        <span class="token punctuation">&#125;</span><span class="token punctuation">)</span></pre></td></tr><tr><td data-num="81"></td><td><pre></pre></td></tr><tr><td data-num="82"></td><td><pre>        self<span class="token punctuation">.</span>ep_obs<span class="token punctuation">,</span> self<span class="token punctuation">.</span>ep_as<span class="token punctuation">,</span> self<span class="token punctuation">.</span>ep_rs <span class="token operator">=</span> <span class="token punctuation">[</span><span class="token punctuation">]</span><span class="token punctuation">,</span> <span class="token punctuation">[</span><span class="token punctuation">]</span><span class="token punctuation">,</span> <span class="token punctuation">[</span><span class="token punctuation">]</span>    <span class="token comment"># empty episode data</span></pre></td></tr><tr><td data-num="83"></td><td><pre><span class="token comment"># Hyper Parameters</span></pre></td></tr><tr><td data-num="84"></td><td><pre>ENV_NAME <span class="token operator">=</span> <span class="token string">'CartPole-v0'</span></pre></td></tr><tr><td data-num="85"></td><td><pre>EPISODE <span class="token operator">=</span> <span class="token number">3000</span> <span class="token comment"># Episode limitation</span></pre></td></tr><tr><td data-num="86"></td><td><pre>STEP <span class="token operator">=</span> <span class="token number">3000</span> <span class="token comment"># Step limitation in an episode</span></pre></td></tr><tr><td data-num="87"></td><td><pre>TEST <span class="token operator">=</span> <span class="token number">10</span> <span class="token comment"># The number of experiment test every 100 episode</span></pre></td></tr><tr><td data-num="88"></td><td><pre></pre></td></tr><tr><td data-num="89"></td><td><pre><span class="token keyword">def</span> <span class="token function">main</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">:</span></pre></td></tr><tr><td data-num="90"></td><td><pre>  <span class="token comment"># initialize OpenAI Gym env and dqn agent</span></pre></td></tr><tr><td data-num="91"></td><td><pre>  env <span class="token operator">=</span> gym<span class="token punctuation">.</span>make<span class="token punctuation">(</span>ENV_NAME<span class="token punctuation">)</span></pre></td></tr><tr><td data-num="92"></td><td><pre>  agent <span class="token operator">=</span> Policy_Gradient<span class="token punctuation">(</span>env<span class="token punctuation">)</span></pre></td></tr><tr><td data-num="93"></td><td><pre></pre></td></tr><tr><td data-num="94"></td><td><pre>  <span class="token keyword">for</span> episode <span class="token keyword">in</span> <span class="token builtin">range</span><span class="token punctuation">(</span>EPISODE<span class="token punctuation">)</span><span class="token punctuation">:</span></pre></td></tr><tr><td data-num="95"></td><td><pre>    <span class="token comment"># initialize task</span></pre></td></tr><tr><td data-num="96"></td><td><pre>    state <span class="token operator">=</span> env<span class="token punctuation">.</span>reset<span class="token punctuation">(</span><span class="token punctuation">)</span></pre></td></tr><tr><td data-num="97"></td><td><pre>    <span class="token comment"># Train</span></pre></td></tr><tr><td data-num="98"></td><td><pre>    <span class="token keyword">for</span> step <span class="token keyword">in</span> <span class="token builtin">range</span><span class="token punctuation">(</span>STEP<span class="token punctuation">)</span><span class="token punctuation">:</span></pre></td></tr><tr><td data-num="99"></td><td><pre>      action <span class="token operator">=</span> agent<span class="token punctuation">.</span>choose_action<span class="token punctuation">(</span>state<span class="token punctuation">)</span> <span class="token comment"># e-greedy action for train</span></pre></td></tr><tr><td data-num="100"></td><td><pre>      next_state<span class="token punctuation">,</span>reward<span class="token punctuation">,</span>done<span class="token punctuation">,</span>_ <span class="token operator">=</span> env<span class="token punctuation">.</span>step<span class="token punctuation">(</span>action<span class="token punctuation">)</span></pre></td></tr><tr><td data-num="101"></td><td><pre>      agent<span class="token punctuation">.</span>store_transition<span class="token punctuation">(</span>state<span class="token punctuation">,</span> action<span class="token punctuation">,</span> reward<span class="token punctuation">)</span></pre></td></tr><tr><td data-num="102"></td><td><pre>      state <span class="token operator">=</span> next_state</pre></td></tr><tr><td data-num="103"></td><td><pre>      <span class="token keyword">if</span> done<span class="token punctuation">:</span></pre></td></tr><tr><td data-num="104"></td><td><pre>        <span class="token comment">#print("stick for ",step, " steps")</span></pre></td></tr><tr><td data-num="105"></td><td><pre>        agent<span class="token punctuation">.</span>learn<span class="token punctuation">(</span><span class="token punctuation">)</span></pre></td></tr><tr><td data-num="106"></td><td><pre>        <span class="token keyword">break</span></pre></td></tr><tr><td data-num="107"></td><td><pre></pre></td></tr><tr><td data-num="108"></td><td><pre>    <span class="token comment"># Test every 100 episodes</span></pre></td></tr><tr><td data-num="109"></td><td><pre>    <span class="token keyword">if</span> episode <span class="token operator">%</span> <span class="token number">100</span> <span class="token operator">==</span> <span class="token number">0</span><span class="token punctuation">:</span></pre></td></tr><tr><td data-num="110"></td><td><pre>      total_reward <span class="token operator">=</span> <span class="token number">0</span></pre></td></tr><tr><td data-num="111"></td><td><pre>      <span class="token keyword">for</span> i <span class="token keyword">in</span> <span class="token builtin">range</span><span class="token punctuation">(</span>TEST<span class="token punctuation">)</span><span class="token punctuation">:</span></pre></td></tr><tr><td data-num="112"></td><td><pre>        state <span class="token operator">=</span> env<span class="token punctuation">.</span>reset<span class="token punctuation">(</span><span class="token punctuation">)</span></pre></td></tr><tr><td data-num="113"></td><td><pre>        <span class="token keyword">for</span> j <span class="token keyword">in</span> <span class="token builtin">range</span><span class="token punctuation">(</span>STEP<span class="token punctuation">)</span><span class="token punctuation">:</span></pre></td></tr><tr><td data-num="114"></td><td><pre>          env<span class="token punctuation">.</span>render<span class="token punctuation">(</span><span class="token punctuation">)</span></pre></td></tr><tr><td data-num="115"></td><td><pre>          action <span class="token operator">=</span> agent<span class="token punctuation">.</span>choose_action<span class="token punctuation">(</span>state<span class="token punctuation">)</span> <span class="token comment"># direct action for test</span></pre></td></tr><tr><td data-num="116"></td><td><pre>          state<span class="token punctuation">,</span>reward<span class="token punctuation">,</span>done<span class="token punctuation">,</span>_ <span class="token operator">=</span> env<span class="token punctuation">.</span>step<span class="token punctuation">(</span>action<span class="token punctuation">)</span></pre></td></tr><tr><td data-num="117"></td><td><pre>          total_reward <span class="token operator">+=</span> reward</pre></td></tr><tr><td data-num="118"></td><td><pre>          <span class="token keyword">if</span> done<span class="token punctuation">:</span></pre></td></tr><tr><td data-num="119"></td><td><pre>            <span class="token keyword">break</span></pre></td></tr><tr><td data-num="120"></td><td><pre>      ave_reward <span class="token operator">=</span> total_reward<span class="token operator">/</span>TEST</pre></td></tr><tr><td data-num="121"></td><td><pre>      <span class="token keyword">print</span> <span class="token punctuation">(</span><span class="token string">'episode: '</span><span class="token punctuation">,</span>episode<span class="token punctuation">,</span><span class="token string">'Evaluation Average Reward:'</span><span class="token punctuation">,</span>ave_reward<span class="token punctuation">)</span></pre></td></tr><tr><td data-num="122"></td><td><pre></pre></td></tr><tr><td data-num="123"></td><td><pre><span class="token keyword">if</span> __name__ <span class="token operator">==</span> <span class="token string">'__main__'</span><span class="token punctuation">:</span></pre></td></tr><tr><td data-num="124"></td><td><pre>  main<span class="token punctuation">(</span><span class="token punctuation">)</span></pre></td></tr></table></figure><p>运行结果：</p><p><img data-src="https://shun309.oss-cn-hangzhou.aliyuncs.com/photos/output.gif" alt=""></p><p><img data-src="https://shun309.oss-cn-hangzhou.aliyuncs.com/photos/20210625113142.png" alt=""></p><h1 id="6-参考"><a class="anchor" href="#6-参考">#</a> 6 参考</h1><p>1.<span class="exturl" data-url="aHR0cHM6Ly93d3cuY25ibG9ncy5jb20vcGluYXJkL3AvMTAxMzc2OTYuaHRtbA=="> 强化学习 (十三) 策略梯度 (Policy Gradient)</span></p><p>2.<span class="exturl" data-url="aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzMwNjE1OTAzL2FydGljbGUvZGV0YWlscy84MDc0NzM4MA==">https://blog.csdn.net/qq_30615903/article/details/80747380</span></p><p>3.<span class="exturl" data-url="aHR0cHM6Ly96aHVhbmxhbi56aGlodS5jb20vcC80MjA1NTExNQ==">https://zhuanlan.zhihu.com/p/42055115</span></p><p>4.<span class="exturl" data-url="aHR0cHM6Ly9tb2ZhbnB5LmNvbQ==">https://mofanpy.com</span></p></div><footer><div class="meta"><span class="item"><span class="icon"><i class="ic i-calendar-check"></i> </span><span class="text">更新于</span> <time title="修改时间：2023-01-12 12:19:43" itemprop="dateModified" datetime="2023-01-12T12:19:43+08:00">2023-01-12</time> </span><span id="posts/898b5689/" class="item leancloud_visitors" data-flag-title="强化学习之 Policy Gradient" title="阅读次数"><span class="icon"><i class="ic i-eye"></i> </span><span class="text">阅读次数</span> <span class="leancloud-visitors-count"></span> <span class="text">次</span></span></div><div class="reward"><button><i class="ic i-heartbeat"></i> 赞赏</button><p>请我喝[茶]~(￣▽￣)~*</p><div id="qr"><div><img data-src="/images/wechatpay.png" alt="hang shun 微信支付"><p>微信支付</p></div><div><img data-src="/images/alipay.png" alt="hang shun 支付宝"><p>支付宝</p></div><div><img data-src="/images/paypal.png" alt="hang shun 贝宝"><p>贝宝</p></div></div></div><div id="copyright"><ul><li class="author"><strong>本文作者： </strong>hang shun <i class="ic i-at"><em>@</em></i>航 順</li><li class="link"><strong>本文链接：</strong> <a href="https://jiang-hs.gitee.io/posts/898b5689/" title="强化学习之 Policy Gradient">https://jiang-hs.gitee.io/posts/898b5689/</a></li><li class="license"><strong>版权声明： </strong>本站所有文章除特别声明外，均采用 <span class="exturl" data-url="aHR0cHM6Ly9jcmVhdGl2ZWNvbW1vbnMub3JnL2xpY2Vuc2VzL2J5LW5jLXNhLzQuMC9kZWVkLnpo"><i class="ic i-creative-commons"><em>(CC)</em></i>BY-NC-SA</span> 许可协议。转载请注明出处！</li></ul></div></footer></article></div><div class="post-nav"><div class="item left"><a href="/posts/ae4d52e6/" itemprop="url" rel="prev" data-background-image="https:&#x2F;&#x2F;pic1.imgdb.cn&#x2F;item&#x2F;64427c6f0d2dde5777affbe2.jpg" title="顺序价格机制的强化学习"><span class="type">上一篇</span> <span class="category"><i class="ic i-flag"></i></span><h3>顺序价格机制的强化学习</h3></a></div><div class="item right"><a href="/posts/7c4ca347/" itemprop="url" rel="next" data-background-image="https:&#x2F;&#x2F;pic1.imgdb.cn&#x2F;item&#x2F;60d7f97b5132923bf8a9b51e.jpg" title="计算机网络概述"><span class="type">下一篇</span> <span class="category"><i class="ic i-flag"></i></span><h3>计算机网络概述</h3></a></div></div><div class="wrap" id="comments"></div></div><div id="sidebar"><div class="inner"><div class="panels"><div class="inner"><div class="contents panel pjax" data-title="文章目录"><ol class="toc"><li class="toc-item toc-level-1"><a class="toc-link" href="#1-%E4%BB%80%E4%B9%88%E6%98%AFpolicy"><span class="toc-number">1.</span> <span class="toc-text">1 什么是 Policy</span></a></li><li class="toc-item toc-level-1"><a class="toc-link" href="#2-%E7%AE%97%E6%B3%95%E6%80%9D%E6%83%B3"><span class="toc-number">2.</span> <span class="toc-text">2 算法思想</span></a></li><li class="toc-item toc-level-1"><a class="toc-link" href="#3-%E7%AD%96%E7%95%A5%E5%87%BD%E6%95%B0"><span class="toc-number">3.</span> <span class="toc-text">3 策略函数</span></a></li><li class="toc-item toc-level-1"><a class="toc-link" href="#4-%E7%AD%96%E7%95%A5%E6%A2%AF%E5%BA%A6%E7%9A%84%E4%BC%98%E5%8C%96%E7%9B%AE%E6%A0%87"><span class="toc-number">4.</span> <span class="toc-text">4 策略梯度的优化目标</span></a></li><li class="toc-item toc-level-1"><a class="toc-link" href="#5-%E4%BB%A3%E7%A0%81%E5%AE%9E%E7%8E%B0"><span class="toc-number">5.</span> <span class="toc-text">5 代码实现</span></a></li><li class="toc-item toc-level-1"><a class="toc-link" href="#6-%E5%8F%82%E8%80%83"><span class="toc-number">6.</span> <span class="toc-text">6 参考</span></a></li></ol></div><div class="related panel pjax" data-title="系列文章"></div><div class="overview panel" data-title="站点概览"><div class="author" itemprop="author" itemscope itemtype="http://schema.org/Person"><img class="image" itemprop="image" alt="hang shun" data-src="/images/avatar.jpg"><p class="name" itemprop="name">hang shun</p><div class="description" itemprop="description">世中逢尔，雨中逢花</div></div><nav class="state"><div class="item posts"><a href="/archives/"><span class="count">45</span> <span class="name">文章</span></a></div><div class="item categories"><a href="/categories/"><span class="count">10</span> <span class="name">分类</span></a></div><div class="item tags"><a href="/tags/"><span class="count">25</span> <span class="name">标签</span></a></div></nav><div class="social"><span class="exturl item github" data-url="aHR0cHM6Ly9naXRodWIuY29tL0pJQU5HLUhT" title="https:&#x2F;&#x2F;github.com&#x2F;JIANG-HS"><i class="ic i-github"></i></span> <span class="exturl item zhihu" data-url="aHR0cHM6Ly93d3cuemhpaHUuY29tL3Blb3BsZS9odWktc2h1bi14aW4tbGl1" title="https:&#x2F;&#x2F;www.zhihu.com&#x2F;people&#x2F;hui-shun-xin-liu"><i class="ic i-zhihu"></i></span> <span class="exturl item music" data-url="aHR0cHM6Ly9tdXNpYy4xNjMuY29tLyMvdXNlci9ob21lP2lkPTE4MzkwMTczMzI=" title="https:&#x2F;&#x2F;music.163.com&#x2F;#&#x2F;user&#x2F;home?id&#x3D;1839017332"><i class="ic i-cloud-music"></i></span> <span class="exturl item bilibili" data-url="aHR0cHM6Ly9zcGFjZS5iaWxpYmlsaS5jb20vMzIxMTYyNDg1" title="https:&#x2F;&#x2F;space.bilibili.com&#x2F;321162485"><i class="ic i-bilibili"></i></span></div><ul class="menu"><li class="item"><a href="/" rel="section"><i class="ic i-home"></i>首页</a></li><li class="item"><a href="/about/" rel="section"><i class="ic i-user"></i>关于</a></li><li class="item dropdown"><a href="javascript:void(0);"><i class="ic i-feather"></i>文章</a><ul class="submenu"><li class="item"><a href="/archives/" rel="section"><i class="ic i-list-alt"></i>归档</a></li><li class="item"><a href="/categories/" rel="section"><i class="ic i-th"></i>分类</a></li><li class="item"><a href="/tags/" rel="section"><i class="ic i-tags"></i>标签</a></li></ul></li><li class="item"><a href="/friends/" rel="section"><i class="ic i-heart"></i>友達</a></li><li class="item"><a href="/movie/" rel="section"><i class="ic i-play"></i>movie</a></li><li class="item"><a href="/music/" rel="section"><i class="ic i-music"></i>music</a></li></ul></div></div></div><ul id="quick"><li class="prev pjax"><a href="/posts/ae4d52e6/" rel="prev" title="上一篇"><i class="ic i-chevron-left"></i></a></li><li class="up"><i class="ic i-arrow-up"></i></li><li class="down"><i class="ic i-arrow-down"></i></li><li class="next pjax"><a href="/posts/7c4ca347/" rel="next" title="下一篇"><i class="ic i-chevron-right"></i></a></li><li class="percent"></li></ul></div></div><div class="dimmer"></div></div></main><footer id="footer"><div class="inner"><div class="widgets"><div class="rpost pjax"><h2>随机文章</h2><ul><li class="item"><div class="breadcrumb"><a href="/categories/%E6%9C%BA%E5%99%A8%E5%AD%A6%E4%B9%A0%E5%9F%BA%E7%A1%80/" title="分类于 机器学习基础">机器学习基础</a></div><span><a href="/posts/2afaae3d/" title="K-means算法">K-means算法</a></span></li><li class="item"><div class="breadcrumb"></div><span><a href="/posts/9dc5ce37/" title="DIB-PEB-Sequientia-RS笔记">DIB-PEB-Sequientia-RS笔记</a></span></li><li class="item"><div class="breadcrumb"></div><span><a href="/posts/4ad97fdc/" title="在jupyter中添加任意环境的kernel">在jupyter中添加任意环境的kernel</a></span></li><li class="item"><div class="breadcrumb"></div><span><a href="/posts/898b5689/" title="强化学习之Policy Gradient">强化学习之Policy Gradient</a></span></li><li class="item"><div class="breadcrumb"></div><span><a href="/posts/9eceac59/" title="基于时空注意力的地点推荐模型学习笔记">基于时空注意力的地点推荐模型学习笔记</a></span></li><li class="item"><div class="breadcrumb"></div><span><a href="/posts/67decac2/" title="初识GCN">初识GCN</a></span></li><li class="item"><div class="breadcrumb"><a href="/categories/%E6%9C%BA%E5%99%A8%E5%AD%A6%E4%B9%A0%E5%9F%BA%E7%A1%80/" title="分类于 机器学习基础">机器学习基础</a></div><span><a href="/posts/c6767314/" title="卷积神经网络">卷积神经网络</a></span></li><li class="item"><div class="breadcrumb"></div><span><a href="/posts/748f0bbe/" title="强化学习">强化学习</a></span></li><li class="item"><div class="breadcrumb"><a href="/categories/%E6%9C%BA%E5%99%A8%E5%AD%A6%E4%B9%A0%E5%9F%BA%E7%A1%80/" title="分类于 机器学习基础">机器学习基础</a></div><span><a href="/posts/20417848/" title="循环神经网络">循环神经网络</a></span></li><li class="item"><div class="breadcrumb"><a href="/categories/%E8%AE%BA%E6%96%87%E7%B2%BE%E8%AF%BB/" title="分类于 论文精读">论文精读</a> <i class="ic i-angle-right"></i> <a href="/categories/%E8%AE%BA%E6%96%87%E7%B2%BE%E8%AF%BB/%E6%96%B0%E9%97%BB%E6%8E%A8%E8%8D%90/" title="分类于 新闻推荐">新闻推荐</a> <i class="ic i-angle-right"></i> <a href="/categories/%E8%AE%BA%E6%96%87%E7%B2%BE%E8%AF%BB/%E6%96%B0%E9%97%BB%E6%8E%A8%E8%8D%90/%E9%9A%90%E7%A7%81%E4%BF%9D%E6%8A%A4%E6%96%B0%E9%97%BB%E6%8E%A8%E8%8D%90/" title="分类于 隐私保护新闻推荐">隐私保护新闻推荐</a> <i class="ic i-angle-right"></i> <a href="/categories/%E8%81%94%E9%82%A6%E5%AD%A6%E4%B9%A0/" title="分类于 联邦学习">联邦学习</a></div><span><a href="/posts/ef078385/" title="FedNewsRec：隐私保护新闻推荐模型学习">FedNewsRec：隐私保护新闻推荐模型学习</a></span></li></ul></div><div><h2>最新评论</h2><ul class="leancloud-recent-comment"></ul></div></div><div class="status"><div class="copyright">&copy; 2020 – <span itemprop="copyrightYear">2023</span> <span class="with-love"><i class="ic i-sakura rotate"></i> </span><span class="author" itemprop="copyrightHolder">hang shun @ hang shun</span></div><div class="count"><span class="post-meta-item-icon"><i class="ic i-chart-area"></i> </span><span title="站点总字数">267k 字</span> <span class="post-meta-divider">|</span> <span class="post-meta-item-icon"><i class="ic i-coffee"></i> </span><span title="站点阅读时长">4:02</span></div><div class="powered-by">基于 <span class="exturl" data-url="aHR0cHM6Ly9oZXhvLmlv">Hexo</span> & Theme.<span class="exturl" data-url="aHR0cHM6Ly9naXRodWIuY29tL2FtZWhpbWUvaGV4by10aGVtZS1zaG9rYQ==">Shoka</span></div></div></div></footer></div><script data-config type="text/javascript">var LOCAL={path:"posts/898b5689/",favicon:{show:"(´Д｀)被发现了！",hide:"（●´3｀●）我藏好了~"},search:{placeholder:"文章搜索",empty:"关于 「 ${query} 」，什么也没搜到",stats:"${time} ms 内找到 ${hits} 条结果"},valine:!0,copy_tex:!0,katex:!0,fancybox:!0,copyright:'复制成功，转载请遵守 <i class="ic i-creative-commons"></i>BY-NC-SA 协议。',ignores:[function(e){return e.includes("#")},function(e){return new RegExp(LOCAL.path+"$").test(e)}]}</script><script src="https://cdn.polyfill.io/v2/polyfill.js"></script><script src="//cdn.jsdelivr.net/combine/npm/pace-js@1.0.2/pace.min.js,npm/pjax@0.2.8/pjax.min.js,npm/whatwg-fetch@3.4.0/dist/fetch.umd.min.js,npm/animejs@3.2.0/lib/anime.min.js,npm/algoliasearch@4/dist/algoliasearch-lite.umd.js,npm/instantsearch.js@4/dist/instantsearch.production.min.js,npm/lozad@1/dist/lozad.min.js,npm/quicklink@2/dist/quicklink.umd.js"></script><script src="/js/app.js?v=0.0.0"></script></body></html><!-- rebuild by hrmmi -->